# -*- coding: utf-8 -*-

import redis
from HifoFzz.utils.redis_pool import POOL, PROXY_REDIS_KEY
import pandas as pd
from HifoFzz.utils.pcd import PCD_LIST
from HifoFzz.utils.mongoclient import MongodbClient
from HifoFzz.items import OfficeItem, ShopsItem, WarehouseItem, WorkshopItem, LandItem, ParkingItem
from HifoFzz.items import LandChinaItem, LandChinaGivingItem
import datetime
import re
from HifoFzz.utils.redis_case_filter import get_filter_flag
import logging

logger = logging.getLogger(__name__)


class RedisConnPipeline(object):
    def open_spider(self, spider):
        spider.redis_conn = redis.StrictRedis(connection_pool=POOL)
        spider.redis_conn.delete(PROXY_REDIS_KEY)

    def close_spider(self, spider):
        spider.redis_conn.close()


class PcdPipeline(object):
    def open_spider(self, spider):
        # 直接解析list，可以用scrapyd部署
        spider.pcd_df = pd.DataFrame(PCD_LIST)

    def close_spider(self, spider):
        del spider.pcd_df


class MongoClientPipeline(object):
    def open_spider(self, spider):
        self.clo = MongodbClient()

    def close_spider(self, spider):
        self.clo.close()

    def process_item(self, item, spider):
        # 贝壳
        if spider.name == 'beike':
            self.clo.set_db('newhours')
            if isinstance(item, OfficeItem):
                self.clo.set_col('beike_office')
                self.clo.insert_one(dict(item))
            if isinstance(item, ShopsItem):
                self.clo.set_col('beike_shops')
                self.clo.insert_one(dict(item))
        if spider.name == 'beike_v2':
            self.clo.set_db('newhours')
            if isinstance(item, OfficeItem):
                self.clo.set_col('beike_office')
                filter_item = dict()
                filter_item['caseId'] = item['caseId']
                self.clo.update_one(filter_item, dict(item))
            if isinstance(item, ShopsItem):
                self.clo.set_col('beike_shops')
                filter_item = dict()
                filter_item['caseId'] = item['caseId']
                self.clo.update_one(filter_item, dict(item))
        # 安居客
        if spider.name == 'anjuke':
            self.clo.set_db('newhours')
            if isinstance(item, OfficeItem):
                self.clo.set_col('anjuke_office')
                self.clo.insert_one(dict(item))
            elif isinstance(item, ShopsItem):
                self.clo.set_col('anjuke_shops')
                self.clo.insert_one(dict(item))
            elif isinstance(item, WarehouseItem):
                self.clo.set_col('anjuke_warehouse')
                self.clo.insert_one(dict(item))
            elif isinstance(item, WorkshopItem):
                self.clo.set_col('anjuke_workshop')
                self.clo.insert_one(dict(item))
            elif isinstance(item, LandItem):
                self.clo.set_col('anjuke_land')
                self.clo.insert_one(dict(item))
            elif isinstance(item, ParkingItem):
                self.clo.set_col('anjuke_parking')
                self.clo.insert_one(dict(item))
        # landchina
        if spider.name == 'landchina':
            self.clo.set_db('newhours')
            if isinstance(item, LandChinaItem):
                self.clo.set_col('result_notice')
                self.clo.update_one({'gdGuid': item['gdGuid']}, dict(item))
            if isinstance(item, LandChinaGivingItem):
                self.clo.set_col('giving_notice')
                self.clo.update_one({'gyggGuid': item['gyggGuid'], 'zdBh': item['zdBh']}, dict(item))
        # 86写字楼
        if spider.name == 'office86':
            self.clo.set_db('newhours')
            if isinstance(item, OfficeItem):
                self.clo.set_col('office86_office')
                self.clo.insert_one(dict(item))


class FzzPipeline(object):
    def process_item(self, item, spider):
        spider_name_list = ['beike', 'anjuke', 'office86', 'beike_v2']
        if any([spider.name == name for name in spider_name_list]):
            item['updateTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            item['fetchTime'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        return item


# 贝壳
class BeikePipeline(object):
    def open_spider(self, spider):
        if spider.name == 'beike':
            case_id_filter = spider.settings['CASE_ID_FILTER'] if spider.settings['CASE_ID_FILTER'] != None else True
            if case_id_filter:
                logger.info(f'>>>>{spider.name}:案例id去重开启<<<<')
                # todo 连接mongo集群需优化
                col_name_li: list = ['beike_shops', 'beike_office', ]
                spider.filter_flag = get_filter_flag('newhours', col_name_li, 'beike_case_id',
                                                     mongo_client=MongodbClient()._client)
                logger.info(f'>>>>{spider.name}:案例id去重配置成功<<<<') if spider.filter_flag else logger.error(
                    f'>>>>{spider.name}:案例id去重配置失败<<<<')
            else:
                spider.filter_flag = None
                logger.info(f'>>>>{spider.name}:不根据案例id进行去重<<<<')

    def process_item(self, item, spider):
        if spider.name == 'beike':
            # shangQuan字段处理
            item['shangQuan'] = self.deal_list(item['shangQuan'])
            # str空值处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item(item_tup[1])
            # highQualityHouse字段处理
            item['highQualityHouse'] = True if item['highQualityHouse'] else False
            # 写字楼字段处理
            if isinstance(item, OfficeItem):
                pass
            # 商铺字段处理
            if isinstance(item, ShopsItem):
                # isStreet字段处理
                item['isStreet'] = True if item['isStreet'] else False
                # operationType字段处理
                item['operationType'] = self.deal_list(item['operationType'])
                # form字段处理
                item['form'] = self.deal_list(item['form'])
                # houseAdvantages字段处理
                item['houseAdvantages'] = item['houseAdvantages'] if item['houseAdvantages'] else None
            item['sr'] = 'Beike'
            item['siteName'] = '贝壳'
        if spider.name == 'beike_v2':
            # str空值处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item(item_tup[1])
            item['sr'] = 'Beike'
            item['siteName'] = '贝壳'
        return item

    @staticmethod
    def deal_item(item_val):
        """
        写字楼&商铺 案例字段str空值处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, str)
            item_val = item_val.strip() if item_val.strip() else None
        except:
            return item_val
        else:
            return item_val

    @staticmethod
    def deal_list(item_val):
        """
        list转换为str
        :param item_val:
        :return:
        """
        try:
            assert isinstance(item_val, list)
            ret = '、'.join(item_val) if len(item_val) else None
        except:
            return item_val
        else:
            return ret


# 安居客
class AnjukePipeline(object):
    def open_spider(self, spider):
        if spider.name == 'anjuke':
            case_id_filter = spider.settings['CASE_ID_FILTER'] if spider.settings['CASE_ID_FILTER'] != None else True
            if case_id_filter:
                logger.info(f'>>>>{spider.name}:案例id去重开启<<<<')
                # todo 连接mongo集群需优化
                col_name_li: list = ['anjuke_shops', 'anjuke_office', 'anjuke_warehouse', 'anjuke_workshop',
                                     'anjuke_land', 'anjuke_parking']
                spider.filter_flag = get_filter_flag('newhours', col_name_li, 'anjuke_case_id',
                                                     mongo_client=MongodbClient()._client)
                logger.info(f'>>>>{spider.name}:案例id去重配置成功<<<<') if spider.filter_flag else logger.error(
                    f'>>>>{spider.name}:案例id去重配置失败<<<<')
            else:
                spider.filter_flag = None
                logger.info(f'>>>>{spider.name}:不根据案例id进行去重<<<<')

    def process_item(self, item, spider):
        if spider.name == 'anjuke':
            # buildArea字段处理
            item['buildArea'] = self.deal_list(item['buildArea'])
            # listingDate字段处理
            item['listingDate'] = self.deal_specical_char(item['listingDate'])
            if '出租' in item['category']:
                # dailyRental字段处理
                item['dailyRental'] = self.deal_list(item['dailyRental'])
                # monthlyRental字段处理
                item['monthlyRental'] = self.deal_list(item['monthlyRental'])
            elif '出售' in item['category']:
                # price字段处理
                item['price'] = self.deal_list(item['price'])
                # totalPrice字段处理
                item['totalPrice'] = self.deal_list(item['totalPrice'])
            elif '转让' in item['category']:
                # monthlyRental字段处理
                item['monthlyRental'] = self.deal_list(item['monthlyRental'])
                # transferFee字段处理
                item['transferFee'] = self.deal_list(item['transferFee'])
            if isinstance(item, ShopsItem):
                # shangQuan字段处理
                item['shangQuan'] = self.deal_specical_char(item['shangQuan'])
            else:
                # locationArea字段处理
                item['locationArea'] = self.deal_specical_char(item['locationArea'])
            item['sr'] = 'AnJuKe'
            item['siteName'] = '安居客'
        return item

    @staticmethod
    def deal_list(item_val):
        """
        list转换为str
        :param item_val:
        :return:
        """
        try:
            assert isinstance(item_val, list)
            ret = ''.join(item_val) if len(item_val) else None
        except:
            return item_val
        else:
            return ret

    @staticmethod
    def deal_specical_char(item_val):
        """
        处理特殊字符
        :param item_val:
        :return:
        """
        regex_1 = re.compile(r'\s+|更新于')
        regex_2 = re.compile(r'-')
        try:
            ret_1 = regex_1.sub('', item_val)
            ret_2 = regex_2.sub('', ret_1, count=1) if ret_1.startswith('-') else ret_1
            ret_2 = ret_2 if ret_2 else None
        except:
            return item_val
        else:
            return ret_2


# landchina
class LandChinaPipeline(object):
    def process_item(self, item, spider):
        if spider.name == 'landchina':
            item['siteName'] = '中国土地市场网'
        if isinstance(item, LandChinaGivingItem):
            item['gyggBh'] = self.get_gyggbh(item['gyggBt'])  # 出让公告编号
            # 特殊字符处理
            for item_tup in item.items():
                item[item_tup[0]] = self.deal_item(item_tup[1])
        return item

    @staticmethod
    def deal_item(item_val):
        """
        案例字段特殊字符处理处理
        :param item_val: 字段的value值
        :return:
        """
        try:
            assert isinstance(item_val, str)
            item_val = item_val.strip() if item_val.strip() else None
        except:
            return item_val
        else:
            return item_val

    @staticmethod
    def get_gyggbh(content: str):
        """
        提取出让公告编号
        :param content:
        :return:
        """
        try:
            ret = content.split('(')[1].replace(')', '')
            assert ret
        except:
            return content
        else:
            return ret


# 86写字楼
class Office86Pipeline(object):
    def open_spider(self, spider):
        if spider.name == 'office86':
            case_id_filter = spider.settings['CASE_ID_FILTER'] if spider.settings['CASE_ID_FILTER'] != None else True
            if case_id_filter:
                logger.info(f'>>>>{spider.name}:案例id去重开启<<<<')
                # todo 连接mongo集群需优化
                col_name_li: list = ['office86_office', ]
                spider.filter_flag = get_filter_flag('newhours', col_name_li, 'office86_case_id',
                                                     mongo_client=MongodbClient()._client)
                logger.info(f'>>>>{spider.name}:案例id去重配置成功<<<<') if spider.filter_flag else logger.error(
                    f'>>>>{spider.name}:案例id去重配置失败<<<<')
            else:
                spider.filter_flag = None
                logger.info(f'>>>>{spider.name}:不根据案例id进行去重<<<<')

    def process_item(self, item, spider):
        if spider.name == 'office86':
            # buildArea字段处理
            item['buildArea'] = self.deal_list(item['buildArea'])
            # elevator字段处理
            item['elevator'] = self.deal_list(item['elevator'])
            # projectBrief字段处理
            item['projectBrief'] = [i.strip() for i in item['projectBrief'] if i and i.strip()]
            if '出租' in item['category']:
                # dailyRental字段处理
                item['dailyRental'] = self.deal_list(item['dailyRental'])
            else:
                # price字段处理
                item['price'] = self.deal_list(item['price'])
        return item

    @staticmethod
    def deal_list(item_val):
        """
        list转换为str
        :param item_val:
        :return:
        """
        try:
            assert isinstance(item_val, list)
            item_val = [i.strip() for i in item_val if i and i.strip()]
            ret = ' '.join(item_val) if len(item_val) else None
        except:
            return item_val
        else:
            return ret


if __name__ == '__main__':
    pcd_df = pd.DataFrame(PCD_LIST)
    city_df = pcd_df.loc[pcd_df.name.str.contains('深圳')]
    city_li = city_df.to_dict(orient='records')
    city_id = city_li[0]['id']
    print(city_id)
